#!/usr/bin/python3
import requests
import json
import pymysql
import time
import random


db = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='root', db='zhihu', charset='utf8')
cursor = db.cursor()
 
def saveQuestionDB(id, title,topic, url):
    #插入数据
    try:
        sql = "insert into questions values (%s,'%s','%s','%s')"%(id, title,topic,url)
        cursor.execute(sql)
        db.commit()
    except Exception as e:
        db.rollback()
        print(e)
 
def saveArticleDB(id, title,excerpt,content,imgurl,topic, vote, cmts, auth, url):
    #插入数据
    try:
        sql = "insert into article values (%s,'%s','%s','%s','%s','%s',%s,%s,'%s', '%s')"%(id, title,excerpt, content,imgurl,topic, vote, cmts, auth, url)
        cursor.execute(sql)
        db.commit()
    except Exception as e:
        db.rollback()
        print(e)
#_zap=2dc31fbe-0b47-441a-8281-2de65726f46d; d_c0=""; __snaker__id=df5uh1b9C3gJMrLr; _9755xjdesxxd_=32; YD00517437729195:WM_NI=rSNQW+kfkx9q8bU6h6VF9pNrJjL6iwkUhFwM1Gn1Q6Ul/EDxZdjddNZG7zh0exvMlguGQMM85fFTImqA74Rj57kUkGYtiBE1JPpRFKNmkdmAQ1qWUZv1rbOYzK333SzSNEE=; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1624878103,1624895971,1624936666,1624936690; captcha_session_v2="2|1:0|10:1624936813|18:captcha_session_v2|88:Mkk3YWRGSlZhdGc5SEhiTjJKN3NWekVnTHFIbFM0M2FKdUVSVkpDYUJRYUFXOEx5eVZqUHBXaUttYnFmdjlnbQ==|4e4ebad68ccd05a914fc5660d2be3a5a457b5b2bba0678009c359d59d21b18c2"; z_c0="2|1:0|10:1624936840|4:z_c0|92:Mi4xN3JneEdnQUFBQUFBc0p5ZHZMdElFeWNBQUFDRUFsVk5oeDRDWVFBUXNXQnR1TTBjVEZPSi1fOU5jajVoaTFDbFBB|7513cfe4dad2f260913249a4ca9d6f19d599cecde10fec25aca7fd05a7c63d43"; tst=r; gdxidpyhxdE=y\IOu3p2TPELuNCto4j0axxDPw3HHQ38s/a1Mo41H+JhSKwVst7mVkZntaQ63giaY+yPc+yqIRLEw\kcAx8XKXecpfArrfVphqfxkP+2CrHLsM2XBZSjMDmKGJ3SRthmy4c\AZ3rwvJggtS83tkL\\N6AgUKANHUI867r7D1BlGKht6+G:1624939250931; SESSIONID=ypIibfIAArEJw72BmGQTNOj2L9buCzOtUSyJbYBbiRe;  KLBRSID=57358d62405ef24305120316801fd92a|1624939668|1624936664
def fetchHotel(url):
    # 发起网络请求，获取数据
    headers = {
        'authority':'www.zhihu.com',
        'method': 'GET',
        'path': '/',
        'scheme': 'https',
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'cache-control': 'max-age=0',
        #'cookie': '_zap=2dc31fbe-0b47-441a-8281-2de65726f46d; d_c0="ALCcnby7SBOPTj592BLLi3cWPBnmTn7JU-I=|1624078566"; __snaker__id=df5uh1b9C3gJMrLr; _9755xjdesxxd_=32; YD00517437729195%3AWM_NI=rSNQW%2Bkfkx9q8bU6h6VF9pNrJjL6iwkUhFwM1Gn1Q6Ul%2FEDxZdjddNZG7zh0exvMlguGQMM85fFTImqA74Rj57kUkGYtiBE1JPpRFKNmkdmAQ1qWUZv1rbOYzK333SzSNEE%3D; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eea8ee6e8f8ba199ca41f48a8bb6d14b929a8fbbae739596aaaff359edb9e1bae12af0fea7c3b92ab28d82b2d621a9baa599bc7bfcbe9bd7eb3e969af791cd7a8ab684d7ec7da98ba8a7f647f399a2d2d43e97b4e1a6c7749abca685e7469a99f99beb3ca7a8baaeef6db7999faed034f7ba8ab5cc67b39ff9d5d162f7e9a5b6ea4d9baca7a3c83b8a90c0b3c5418eb3a585f04b93efbfb8d852b6efbe83ce53ba95fdd4b259ba8d82a6c837e2a3; YD00517437729195%3AWM_TID=NThT7nU5WjxEAEAFQEIr3WILNtG%2BhvdK; _xsrf=ea3392df-ce91-4066-a26c-6454e5c46437; captcha_session_v2="2|1:0|10:1624936813|18:captcha_session_v2|88:Mkk3YWRGSlZhdGc5SEhiTjJKN3NWekVnTHFIbFM0M2FKdUVSVkpDYUJRYUFXOEx5eVZqUHBXaUttYnFmdjlnbQ==|4e4ebad68ccd05a914fc5660d2be3a5a457b5b2bba0678009c359d59d21b18c2"; z_c0="2|1:0|10:1624936840|4:z_c0|92:Mi4xN3JneEdnQUFBQUFBc0p5ZHZMdElFeWNBQUFDRUFsVk5oeDRDWVFBUXNXQnR1TTBjVEZPSi1fOU5jajVoaTFDbFBB|7513cfe4dad2f260913249a4ca9d6f19d599cecde10fec25aca7fd05a7c63d43"; tst=r; gdxidpyhxdE=y%5CIOu3p2TPELuNCto4j0axxDPw3HHQ38s%2Fa1Mo41H%2BJhSKwVst7mVkZntaQ63giaY%2ByPc%2ByqIRLEw%5CkcAx8XKXecpfArrfVphqfxkP%2B2CrHLsM2XBZSjMDmKGJ3SRthmy4c%5CAZ3rwvJggtS83tkL%5CN6AgUKANHUI867r7D1BlGKht6%2BG%3A1624939250931; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1624940212,1624940225,1624940242,1624940887; unlock_ticket="AIDcRll-CBEnAAAAhAJVTRqt2mA8sIUGocyo7iax6VPKFOkdphJPZA=="; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1624943102; SESSIONID=iq86VbL22If1RQ4DniskOgPJR2KJN2qUIIMLBNyIS5X; KLBRSID=57358d62405ef24305120316801fd92a|1624943103|1624936664; JOID=VFkWAUnhMOxAEXLwTOWzNKT4n6daj0PVdkMXvn_XfohwQyfDChuaNyQSd_NNwKgb4bPq2peOCz51PEwUYSnBQmM=; osd=UlgQBUnnMepEEXTxSuGzMqX-m6dcjkXRdkUWuHvXeIl2RyfFCx2eNyITcfdNxqkd5bPs25GKCzh0OkgUZyjHRmM=',
        'cookie':'_zap=2dc31fbe-0b47-441a-8281-2de65726f46d; d_c0="ALCcnby7SBOPTj592BLLi3cWPBnmTn7JU-I=|1624078566"; __snaker__id=df5uh1b9C3gJMrLr; _9755xjdesxxd_=32; YD00517437729195:WM_TID=NThT7nU5WjxEAEAFQEIr3WILNtG+hvdK; _xsrf=eW65Nqt9qbyD5fsNrZNajuGdhzy3y4Fl; tst=r; SESSIONID=NP3nAQA0fHetJtz1ZAc1gEinPpqQQXa92TgEzmNs6Vg; JOID=V1sRAktMDjgfzEkoDkiN5PogpH8cBFkOcIgebjsvQFpuhzhUOA5KRXrPSCoKRVaZajsK-UXbokWF6QxMJmPIpSk=; osd=VVEcAEJOBDUdxUsiA0qE5vAtpnYeDlQMeYoUYzkmQlBjhTFWMgNITHjFRSgDR1yUaDII80jZq0eP5A5FJGnFpyA=; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1633869824,1634382863,1635337477,1635338224; captcha_session_v2="2|1:0|10:1635338241|18:captcha_session_v2|88:dE1PSGJaTTdMR2NabFdnbXE4L2YxZWVxY2pJbEVZNW1HMGpvUEZWVlRXNDhVWXIvdzZxejZZQklpRzVLWTJDRA==|f287ad29cd69493eedddde911c6a0bd32f823ac947cb51df19463359a21fa895"; gdxidpyhxdE=+JRvfTmj95rc/M+y\\nAMk/kRxAxK8wYLJNM57Q9oDWvNsot1BJOAr5ZOutDuS2AOtkOlDsfhX1UtL2SknsetEhdqpP1f81gShktckpTo0NNKxp/c+US+f9J4EL7\jxgIE4N7GBEL4UlEtsHACJSRTs+BfgG1TO7HRBxCh2OwIvgoabfW:1635339143398; YD00517437729195:WM_NI=e8dwnUvew9sbwCxOTZpSGwAQafoYrq25uNknRDVB37r9r4RZoSmlxFWlLK73C+0OnEUOREQs8VuBvLMj9y0iJy1C8sTlsLTd22DM8PGlhn2qS0SmFci2Fer+/ZFX9o+dV2c=; YD00517437729195:WM_NIKE=9ca17ae2e6ffcda170e2e6eeb5dc5987b5a38df63f95ef8bb3c84f928b9b85b5798d9b9cd5c470b0e9a095c42af0fea7c3b92aa7e8ac98ed5fb6bbb9daf26ab59db8b0d752b48d8998c45ba3baa1baf17bf2ebbbafea66929c9aa3b23db2e7bed5d152ad8da38dfb4e95b484acb63981f500b0ed4f97919ea6d479fc8abba3f5488fb6f79ab3638aa7ffa4bb7998b4bdadd770f2ee8caebb5c85b4a88fb17b91f09d8ec569abee8a86f121ed868ab9bc65e9bc9ad3e237e2a3; z_c0="2|1:0|10:1635338270|4:z_c0|92:Mi4xN3JneEdnQUFBQUFBc0p5ZHZMdElFeWNBQUFDRUFsVk5IdFdnWVFDX0pIekp1T2k0WVNQNVhudzdSY2VKQWJ4elhR|0a4cabfb3477827a83c2237a704907a0413cbb1f46f7b0aff58927cda86f0eed"; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1635338273; KLBRSID=d6f775bb0765885473b0cba3a5fa9c12|1635338285|1635337476',
        'sec-fetch-dest': 'document',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-site':'none',
        'sec-fetch-user': '?1',
        'upgrade-insecure-requests':'1',
        #'x-ab-param':'pf_adjust=1;tp_contents=2;zr_expslotpaid=3;se_ffzx_jushen1=0;tp_zrec=1;tp_topic_style=0;qap_question_author=0;top_test_4_liguangyi=1;tp_dingyue_video=0;pf_noti_entry_num=2;qap_question_visitor= 0;zr_slotpaidexp=1',
        #'x-ab-pb':'CrYBCARdBBsAuQLXAkMEJQQ/AMACZwRpAW0CoQNqBDcM1wvkCmwDoAM0DGQEQwBAAYwCtADYAuoD3AsLBJsLtQvCAqIDDwtCBE8BVwRHAGAEwQKrA7cDRQRSC4kCKgPoA+EDCQQOBH0CjgMKBCoEaQT0CzMEeQRgC3QBKgL4A2oBOwKEAjQE9gLPC7QK7AqmATIDcgPzA00EAQtWDMcC9ANoBJ8CygLMAlcDiQyNAU8DUANsBAcM4AsSWwAAABUAAAEAAQACAAAAAQAAAAAAABUBAAAACwAAAgMAAAECAAAAAAEAAAABBAEAAAAAAQABAAEAAAAAAAAAAAAAAAALAAEAAAEAAAABAAAAAQAAAAAAAAAAAQA=',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
    }
 
    # 发起网络请求
    r = requests.get(url,headers=headers)
    r.encoding = 'Unicode'
    return r.text
 
def parseJson(text,topicID):
    json_data = json.loads(text)
    print(json_data)
    lst = json_data['data']
    nextUrl = json_data['paging']['next']
    
    if not lst:
        return;
 
    for item in lst:
        type = item['target']['type']
        
        if type == 'answer':
            # 回答
            question = item['target']['question']
            id = question['id']
            title = question['title']
            url = 'https://www.zhihu.com/question/' + str(id)
            print("问题：",id,title)
            # 保存到数据库
            saveQuestionDB(id,title,topicID,url)
 
        elif type == 'article':
            #专栏
            zhuanlan = item['target']
            #print(zhuanlan)
            id = zhuanlan['id']
            title = zhuanlan['title']
            content = zhuanlan['content']
            excerpt = zhuanlan['excerpt']
            if(len(zhuanlan['topic_thumbnails'])>0):
                imgurl=zhuanlan['topic_thumbnails'][0]
            else:
                imgurl=""
            url = zhuanlan['url']
            vote = zhuanlan['voteup_count']
            cmts = zhuanlan['comment_count']
            auth = zhuanlan['author']['name']
            print("专栏：",id,title)
            # 保存到数据库
            saveArticleDB(id, title,excerpt,content,imgurl,topicID, vote, cmts, auth, url)
 
        elif type == 'question':
            # 问题
            question = item['target']
            id = question['id']
            title = question['title']
            url = 'https://www.zhihu.com/question/' + str(id)
            #373341367   135319431
            print("问题：",id,title)
            # 保存到数据库
            saveQuestionDB(id,title,topicID,url)
 
    return nextUrl
 
if __name__ == '__main__':
    #topicID = '19604694'
    #topicID='19551275'#人工智能
    #topicID='19552449'#电子书
    #topicID='19554982'#笔记
    topicID ='21344535' #阿里云盘

    # 讨论
    url = 'https://www.zhihu.com/api/v4/topics/' + topicID + '/feeds/top_activity?include=data%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.content%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.is_normal%2Ccomment_count%2Cvoteup_count%2Ccontent%2Crelevant_info%2Cexcerpt.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Darticle%29%5D.target.content%2Cvoteup_count%2Ccomment_count%2Cvoting%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Dpeople%29%5D.target.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.annotation_detail%2Ccontent%2Chermes_label%2Cis_labeled%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Canswer_type%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.paid_info%3Bdata%5B%3F%28target.type%3Darticle%29%5D.target.annotation_detail%2Ccontent%2Chermes_label%2Cis_labeled%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dquestion%29%5D.target.annotation_detail%2Ccomment_count%3B&limit=10&after_id=0'
    while url:
       text = fetchHotel(url)
       url = parseJson(text,topicID)
       t=random.random()*10
       time.sleep(t)
 
    # 精华
    url = 'https://www.zhihu.com/api/v4/topics/' + topicID + '/feeds/essence?include=data%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.content%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.is_normal%2Ccomment_count%2Cvoteup_count%2Ccontent%2Crelevant_info%2Cexcerpt.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Darticle%29%5D.target.content%2Cvoteup_count%2Ccomment_count%2Cvoting%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Dpeople%29%5D.target.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.annotation_detail%2Ccontent%2Chermes_label%2Cis_labeled%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Canswer_type%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.paid_info%3Bdata%5B%3F%28target.type%3Darticle%29%5D.target.annotation_detail%2Ccontent%2Chermes_label%2Cis_labeled%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dquestion%29%5D.target.annotation_detail%2Ccomment_count%3B&limit=10&offset=0'
    while url:
       text = fetchHotel(url)
       url = parseJson(text,topicID)
       t=random.random()*10
       time.sleep(t)

 
    # 等待回答
    url = 'https://www.zhihu.com/api/v4/topics/' + topicID + '/feeds/top_question?include=data%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.content%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.is_normal%2Ccomment_count%2Cvoteup_count%2Ccontent%2Crelevant_info%2Cexcerpt.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Darticle%29%5D.target.content%2Cvoteup_count%2Ccomment_count%2Cvoting%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Dpeople%29%5D.target.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.annotation_detail%2Ccontent%2Chermes_label%2Cis_labeled%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Canswer_type%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.paid_info%3Bdata%5B%3F%28target.type%3Darticle%29%5D.target.annotation_detail%2Ccontent%2Chermes_label%2Cis_labeled%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dquestion%29%5D.target.annotation_detail%2Ccomment_count%3B&limit=10&offset=0'
    while url:
       text = fetchHotel(url)
       url = parseJson(text,topicID)
       t=random.random()*10
       time.sleep(t)